{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1fe6e643-9453-4381-9445-bd471685fb96",
   "metadata": {},
   "source": [
    "## Exploring the LEDGAR dataset using Autolabel"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "80110a5b-2b3e-45e2-a2da-f6fa00200dff",
   "metadata": {},
   "source": [
    "#### Setup the API Keys for providers that you want to use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "92993c83-4473-4e05-9510-f543b070c7d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "# provide your own OpenAI API key here\n",
    "os.environ['OPENAI_API_KEY'] = 'sk-xxxxxxxxxxxxxxxxx'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c246f85",
   "metadata": {},
   "source": [
    "#### Install the autolabel library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "bc181e31",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: refuel-autolabel[openai] in /opt/homebrew/lib/python3.10/site-packages (0.0.1)\n",
      "Requirement already satisfied: loguru>=0.5.0 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (0.5.3)\n",
      "Requirement already satisfied: numpy>=1.23.0 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (1.23.3)\n",
      "Requirement already satisfied: requests>=2.27.0 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (2.28.1)\n",
      "Requirement already satisfied: datasets>=2.7.0 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (2.11.0)\n",
      "Requirement already satisfied: langchain>=0.0.190 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (0.0.191)\n",
      "Requirement already satisfied: nervaluate>=0.1.8 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (0.1.8)\n",
      "Requirement already satisfied: pandas>=1.3.0 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (1.5.0)\n",
      "Requirement already satisfied: scikit-learn>=1.0.0 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (1.1.2)\n",
      "Requirement already satisfied: tenacity>=8.2.2 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (8.2.2)\n",
      "Requirement already satisfied: SQLAlchemy==1.4.47 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (1.4.47)\n",
      "Requirement already satisfied: regex>=2023.6.3 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (2023.6.3)\n",
      "Requirement already satisfied: rich>=13.3.5 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (13.4.1)\n",
      "Requirement already satisfied: scipy>=1.10.1 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (1.10.1)\n",
      "Requirement already satisfied: pydantic>=1.10.9 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (1.10.9)\n",
      "Requirement already satisfied: torch>=1.10.0 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (1.13.0)\n",
      "Requirement already satisfied: matplotlib>=3.5.0 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (3.6.0)\n",
      "Requirement already satisfied: wget>=3.2 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (3.2)\n",
      "Requirement already satisfied: openai>=0.27.4 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (0.27.6)\n",
      "Requirement already satisfied: tiktoken>=0.3.3 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (0.3.3)\n",
      "Requirement already satisfied: pyarrow>=8.0.0 in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (9.0.0)\n",
      "Requirement already satisfied: dill<0.3.7,>=0.3.0 in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (0.3.5.1)\n",
      "Requirement already satisfied: tqdm>=4.62.1 in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (4.64.1)\n",
      "Requirement already satisfied: xxhash in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (3.1.0)\n",
      "Requirement already satisfied: multiprocess in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (0.70.13)\n",
      "Requirement already satisfied: fsspec[http]>=2021.11.1 in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (2022.10.0)\n",
      "Requirement already satisfied: aiohttp in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (3.8.3)\n",
      "Requirement already satisfied: huggingface-hub<1.0.0,>=0.11.0 in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (0.14.1)\n",
      "Requirement already satisfied: packaging in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (21.3)\n",
      "Requirement already satisfied: responses<0.19 in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (0.18.0)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (6.0)\n",
      "Requirement already satisfied: async-timeout<5.0.0,>=4.0.0 in /opt/homebrew/lib/python3.10/site-packages (from langchain>=0.0.190->refuel-autolabel[openai]) (4.0.2)\n",
      "Requirement already satisfied: dataclasses-json<0.6.0,>=0.5.7 in /opt/homebrew/lib/python3.10/site-packages (from langchain>=0.0.190->refuel-autolabel[openai]) (0.5.7)\n",
      "Requirement already satisfied: numexpr<3.0.0,>=2.8.4 in /opt/homebrew/lib/python3.10/site-packages (from langchain>=0.0.190->refuel-autolabel[openai]) (2.8.4)\n",
      "Requirement already satisfied: openapi-schema-pydantic<2.0,>=1.2 in /opt/homebrew/lib/python3.10/site-packages (from langchain>=0.0.190->refuel-autolabel[openai]) (1.2.4)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /opt/homebrew/lib/python3.10/site-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (1.0.5)\n",
      "Requirement already satisfied: cycler>=0.10 in /opt/homebrew/lib/python3.10/site-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (0.11.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /opt/homebrew/lib/python3.10/site-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (4.37.3)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /opt/homebrew/lib/python3.10/site-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (1.4.4)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /opt/homebrew/lib/python3.10/site-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (9.2.0)\n",
      "Requirement already satisfied: pyparsing>=2.2.1 in /opt/homebrew/lib/python3.10/site-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (3.0.9)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /opt/homebrew/lib/python3.10/site-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /opt/homebrew/lib/python3.10/site-packages (from pandas>=1.3.0->refuel-autolabel[openai]) (2022.2.1)\n",
      "Requirement already satisfied: typing-extensions>=4.2.0 in /opt/homebrew/lib/python3.10/site-packages (from pydantic>=1.10.9->refuel-autolabel[openai]) (4.3.0)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in /opt/homebrew/lib/python3.10/site-packages (from requests>=2.27.0->refuel-autolabel[openai]) (2.1.1)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/homebrew/lib/python3.10/site-packages (from requests>=2.27.0->refuel-autolabel[openai]) (2.8)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/homebrew/lib/python3.10/site-packages (from requests>=2.27.0->refuel-autolabel[openai]) (1.26.13)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/homebrew/lib/python3.10/site-packages (from requests>=2.27.0->refuel-autolabel[openai]) (2022.9.14)\n",
      "Requirement already satisfied: markdown-it-py<3.0.0,>=2.2.0 in /opt/homebrew/lib/python3.10/site-packages (from rich>=13.3.5->refuel-autolabel[openai]) (2.2.0)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /opt/homebrew/lib/python3.10/site-packages (from rich>=13.3.5->refuel-autolabel[openai]) (2.15.1)\n",
      "Requirement already satisfied: joblib>=1.0.0 in /opt/homebrew/lib/python3.10/site-packages (from scikit-learn>=1.0.0->refuel-autolabel[openai]) (1.2.0)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /opt/homebrew/lib/python3.10/site-packages (from scikit-learn>=1.0.0->refuel-autolabel[openai]) (3.1.0)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /opt/homebrew/lib/python3.10/site-packages (from aiohttp->datasets>=2.7.0->refuel-autolabel[openai]) (22.1.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/homebrew/lib/python3.10/site-packages (from aiohttp->datasets>=2.7.0->refuel-autolabel[openai]) (6.0.2)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /opt/homebrew/lib/python3.10/site-packages (from aiohttp->datasets>=2.7.0->refuel-autolabel[openai]) (1.8.1)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /opt/homebrew/lib/python3.10/site-packages (from aiohttp->datasets>=2.7.0->refuel-autolabel[openai]) (1.3.1)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /opt/homebrew/lib/python3.10/site-packages (from aiohttp->datasets>=2.7.0->refuel-autolabel[openai]) (1.2.0)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: marshmallow<4.0.0,>=3.3.0 in /opt/homebrew/lib/python3.10/site-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain>=0.0.190->refuel-autolabel[openai]) (3.19.0)\n",
      "Requirement already satisfied: marshmallow-enum<2.0.0,>=1.5.1 in /opt/homebrew/lib/python3.10/site-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain>=0.0.190->refuel-autolabel[openai]) (1.5.1)\n",
      "Requirement already satisfied: typing-inspect>=0.4.0 in /opt/homebrew/lib/python3.10/site-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain>=0.0.190->refuel-autolabel[openai]) (0.8.0)\n",
      "Requirement already satisfied: filelock in /opt/homebrew/lib/python3.10/site-packages (from huggingface-hub<1.0.0,>=0.11.0->datasets>=2.7.0->refuel-autolabel[openai]) (3.8.0)\n",
      "Requirement already satisfied: mdurl~=0.1 in /opt/homebrew/lib/python3.10/site-packages (from markdown-it-py<3.0.0,>=2.2.0->rich>=13.3.5->refuel-autolabel[openai]) (0.1.2)\n",
      "Requirement already satisfied: six>=1.5 in /opt/homebrew/lib/python3.10/site-packages (from python-dateutil>=2.7->matplotlib>=3.5.0->refuel-autolabel[openai]) (1.12.0)\n",
      "Requirement already satisfied: mypy-extensions>=0.3.0 in /opt/homebrew/lib/python3.10/site-packages (from typing-inspect>=0.4.0->dataclasses-json<0.6.0,>=0.5.7->langchain>=0.0.190->refuel-autolabel[openai]) (0.4.3)\n"
     ]
    }
   ],
   "source": [
    "!pip install 'refuel-autolabel[openai]'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28d35966",
   "metadata": {},
   "source": [
    "#### Download the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c4b07ab8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/homebrew/lib/python3.10/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading seed example dataset to \"seed.csv\"...\n",
      "100% [........................................................] 139187 / 139187\n",
      "\n",
      "Downloading test dataset to \"test.csv\"...\n",
      "100% [......................................................] 1376364 / 1376364"
     ]
    }
   ],
   "source": [
    "from autolabel import get_data\n",
    "\n",
    "get_data('ledgar')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c13e2536",
   "metadata": {},
   "source": [
    "This downloads two datasets:\n",
    "* `test.csv`: This is the larger dataset we are trying to label using LLMs\n",
    "* `seed.csv`: This is a small dataset where we already have human-provided labels"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "54a676b6",
   "metadata": {},
   "source": [
    "## Start the labeling process!\n",
    "\n",
    "Labeling with Autolabel is a 3-step process:\n",
    "* First, we specify a labeling configuration (see `config.json` below)\n",
    "* Next, we do a dry-run on our dataset using the LLM specified in `config.json` by running `agent.plan`\n",
    "* Finally, we run the labeling with `agent.run`"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "68190738",
   "metadata": {},
   "source": [
    "### First labeling run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c093fe91-3508-4140-8bd6-217034e3cce6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "from autolabel import LabelingAgent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c93fae0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the config\n",
    "with open('config_ledgar.json', 'r') as f:\n",
    "     config = json.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53e0a386",
   "metadata": {},
   "source": [
    "Let's review the configuration file below. You'll notice the following useful keys:\n",
    "* `task_type`: `classification` (since it's a classification task)\n",
    "* `model`: `{'provider': 'openai', 'name': 'gpt-3.5-turbo'}` (use a specific OpenAI model)\n",
    "* `prompt.task_guidelines`: `'You are an expert at understanding legal contracts...` (how we describe the task to the LLM)\n",
    "* `prompt.labels`: `['Agreements',\n",
    "   'Amendments',\n",
    "   'Anti-Corruption Laws',\n",
    "   'Applicable Laws',\n",
    "   'Approvals',\n",
    "   'Arbitration',\n",
    "   'Assignments',\n",
    "   'Assigns', ...]` (the full list of labels to choose from)\n",
    "* `prompt.few_shot_num`: 4 (how many labeled examples to provide to the LLM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8decb1f4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'task_name': 'LegalProvisionsClassification',\n",
       " 'task_type': 'classification',\n",
       " 'dataset': {'label_column': 'label', 'delimiter': ','},\n",
       " 'model': {'provider': 'openai', 'name': 'gpt-3.5-turbo'},\n",
       " 'prompt': {'task_guidelines': 'You are an expert at understanding legal contracts. Your job is to correctly classify legal provisions in contracts into one of the following categories.\\nCategories:{labels}\\n',\n",
       "  'labels': ['Agreements',\n",
       "   'Amendments',\n",
       "   'Anti-Corruption Laws',\n",
       "   'Applicable Laws',\n",
       "   'Approvals',\n",
       "   'Arbitration',\n",
       "   'Assignments',\n",
       "   'Assigns',\n",
       "   'Authority',\n",
       "   'Authorizations',\n",
       "   'Base Salary',\n",
       "   'Benefits',\n",
       "   'Binding Effects',\n",
       "   'Books',\n",
       "   'Brokers',\n",
       "   'Capitalization',\n",
       "   'Change In Control',\n",
       "   'Closings',\n",
       "   'Compliance With Laws',\n",
       "   'Confidentiality',\n",
       "   'Consent To Jurisdiction',\n",
       "   'Consents',\n",
       "   'Construction',\n",
       "   'Cooperation',\n",
       "   'Costs',\n",
       "   'Counterparts',\n",
       "   'Death',\n",
       "   'Defined Terms',\n",
       "   'Definitions',\n",
       "   'Disability',\n",
       "   'Disclosures',\n",
       "   'Duties',\n",
       "   'Effective Dates',\n",
       "   'Effectiveness',\n",
       "   'Employment',\n",
       "   'Enforceability',\n",
       "   'Enforcements',\n",
       "   'Entire Agreements',\n",
       "   'Erisa',\n",
       "   'Existence',\n",
       "   'Expenses',\n",
       "   'Fees',\n",
       "   'Financial Statements',\n",
       "   'Forfeitures',\n",
       "   'Further Assurances',\n",
       "   'General',\n",
       "   'Governing Laws',\n",
       "   'Headings',\n",
       "   'Indemnifications',\n",
       "   'Indemnity',\n",
       "   'Insurances',\n",
       "   'Integration',\n",
       "   'Intellectual Property',\n",
       "   'Interests',\n",
       "   'Interpretations',\n",
       "   'Jurisdictions',\n",
       "   'Liens',\n",
       "   'Litigations',\n",
       "   'Miscellaneous',\n",
       "   'Modifications',\n",
       "   'No Conflicts',\n",
       "   'No Defaults',\n",
       "   'No Waivers',\n",
       "   'Non-Disparagement',\n",
       "   'Notices',\n",
       "   'Organizations',\n",
       "   'Participations',\n",
       "   'Payments',\n",
       "   'Positions',\n",
       "   'Powers',\n",
       "   'Publicity',\n",
       "   'Qualifications',\n",
       "   'Records',\n",
       "   'Releases',\n",
       "   'Remedies',\n",
       "   'Representations',\n",
       "   'Sales',\n",
       "   'Sanctions',\n",
       "   'Severability',\n",
       "   'Solvency',\n",
       "   'Specific Performance',\n",
       "   'Submission To Jurisdiction',\n",
       "   'Subsidiaries',\n",
       "   'Successors',\n",
       "   'Survival',\n",
       "   'Tax Withholdings',\n",
       "   'Taxes',\n",
       "   'Terminations',\n",
       "   'Terms',\n",
       "   'Titles',\n",
       "   'Transactions With Affiliates',\n",
       "   'Use Of Proceeds',\n",
       "   'Vacations',\n",
       "   'Venues',\n",
       "   'Vesting',\n",
       "   'Waiver Of Jury Trials',\n",
       "   'Waivers',\n",
       "   'Warranties',\n",
       "   'Withholdings'],\n",
       "  'example_template': 'Example: {example}\\nOutput: {label}',\n",
       "  'few_shot_examples': 'seed.csv',\n",
       "  'few_shot_selection': 'semantic_similarity',\n",
       "  'few_shot_num': 4}}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "aa3884b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create an agent for labeling\n",
    "agent = LabelingAgent(config=config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "92667a39",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┌──────────────────────────┬─────────┐\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Total Estimated Cost     </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> $8.3316 </span>│\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Number of Examples       </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> 2000    </span>│\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Average cost per example </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> $0.0042 </span>│\n",
       "└──────────────────────────┴─────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┌──────────────────────────┬─────────┐\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mTotal Estimated Cost    \u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m$8.3316\u001b[0m\u001b[1;32m \u001b[0m│\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mNumber of Examples      \u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m2000   \u001b[0m\u001b[1;32m \u001b[0m│\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mAverage cost per example\u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m$0.0042\u001b[0m\u001b[1;32m \u001b[0m│\n",
       "└──────────────────────────┴─────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────────── </span>Prompt Example<span style=\"color: #00ff00; text-decoration-color: #00ff00\"> ──────────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────────── \u001b[0mPrompt Example\u001b[92m ──────────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You are an expert at understanding legal contracts. Your job is to correctly classify legal provisions in contracts into one of the following categories.\n",
      "Categories:Agreements\n",
      "Amendments\n",
      "Anti-Corruption Laws\n",
      "Applicable Laws\n",
      "Approvals\n",
      "Arbitration\n",
      "Assignments\n",
      "Assigns\n",
      "Authority\n",
      "Authorizations\n",
      "Base Salary\n",
      "Benefits\n",
      "Binding Effects\n",
      "Books\n",
      "Brokers\n",
      "Capitalization\n",
      "Change In Control\n",
      "Closings\n",
      "Compliance With Laws\n",
      "Confidentiality\n",
      "Consent To Jurisdiction\n",
      "Consents\n",
      "Construction\n",
      "Cooperation\n",
      "Costs\n",
      "Counterparts\n",
      "Death\n",
      "Defined Terms\n",
      "Definitions\n",
      "Disability\n",
      "Disclosures\n",
      "Duties\n",
      "Effective Dates\n",
      "Effectiveness\n",
      "Employment\n",
      "Enforceability\n",
      "Enforcements\n",
      "Entire Agreements\n",
      "Erisa\n",
      "Existence\n",
      "Expenses\n",
      "Fees\n",
      "Financial Statements\n",
      "Forfeitures\n",
      "Further Assurances\n",
      "General\n",
      "Governing Laws\n",
      "Headings\n",
      "Indemnifications\n",
      "Indemnity\n",
      "Insurances\n",
      "Integration\n",
      "Intellectual Property\n",
      "Interests\n",
      "Interpretations\n",
      "Jurisdictions\n",
      "Liens\n",
      "Litigations\n",
      "Miscellaneous\n",
      "Modifications\n",
      "No Conflicts\n",
      "No Defaults\n",
      "No Waivers\n",
      "Non-Disparagement\n",
      "Notices\n",
      "Organizations\n",
      "Participations\n",
      "Payments\n",
      "Positions\n",
      "Powers\n",
      "Publicity\n",
      "Qualifications\n",
      "Records\n",
      "Releases\n",
      "Remedies\n",
      "Representations\n",
      "Sales\n",
      "Sanctions\n",
      "Severability\n",
      "Solvency\n",
      "Specific Performance\n",
      "Submission To Jurisdiction\n",
      "Subsidiaries\n",
      "Successors\n",
      "Survival\n",
      "Tax Withholdings\n",
      "Taxes\n",
      "Terminations\n",
      "Terms\n",
      "Titles\n",
      "Transactions With Affiliates\n",
      "Use Of Proceeds\n",
      "Vacations\n",
      "Venues\n",
      "Vesting\n",
      "Waiver Of Jury Trials\n",
      "Waivers\n",
      "Warranties\n",
      "Withholdings\n",
      "\n",
      "\n",
      "You will return the answer with just one element: \"the correct label\"\n",
      "\n",
      "Some examples with their output answers are provided below:\n",
      "\n",
      "Example: The Company hereby employs Executive and Executive accepts employment under the terms and conditions of this Agreement.\n",
      "Output: Employment\n",
      "\n",
      "Example: This Agreement, including its attached Exhibit A, constitutes the entire employment agreement between Executive and the Company regarding the terms and conditions of his employment. This Agreement supersedes all prior negotiations, representations or agreements between Executive and the Company, whether written or oral, concerning Executive’s employment.\n",
      "Output: Entire Agreements\n",
      "\n",
      "Example: Executive agrees that during the Employment Period, Executive will devote his/her full business time, energies and talents to serving as the Senior Vice President Community Banking of the Employer, at the direction of the Chief Executive Officer of the Employwer  (the “ CEO ”).  Executive shall have such duties and responsibilities as may be assigned to Executive from time to time by the CEO, which duties and responsibilities shall be commensurate with Executive’s position, shall perform all duties assigned to Executive faithfully and efficiently, subject to the direction of the CEO, and shall have such authorities and powers as are inherent to the undertakings applicable to Executive’s position and necessary to carry out the responsibilities and duties required of Executive hereunder.  Executive will perform the duties required by this Agreement at the Company’s principal place of business unless the nature of such duties requires otherwise.  Notwithstanding the foregoing, during the Employment Period, Executive may devote reasonable time to activities other than those required under this Agreement, including activities of a charitable, educational, religious or similar nature (including professional associations) to the extent such activities do not, in the reasonable judgment of the CEO, inhibit, prohibit, interfere with or conflict with Executive’s duties under this Agreement or conflict in any material way with the business of the Employer and its Affiliates; provided, however, that Executive shall not serve on the board of directors of any business (other than the Employer or its Affiliates) or hold any other position with any business without receiving the prior written consent of the CEO.\n",
      "Output: Duties\n",
      "\n",
      "Example: Executive represents that Executive's employment by the Company and the performance by Executive of her obligations under this Agreement do not, and shall not, breach any agreement, including, but not limited to, any agreement that obligates him to keep in confidence any trade secrets or confidential or proprietary information of her or of any other party, to perform services for any other party or to refrain from competing, directly or indirectly, with the business of any other party. Executive shall not disclose to the Company or use any trade secrets or confidential or proprietary information of any other party.\n",
      "Output: Representations\n",
      "\n",
      "Now I want you to label the following example:\n",
      "Example: Executive agrees to be employed with the Company, and the Company agrees to employ Executive, during the Term and on the terms and conditions set forth in this Agreement. Executive agrees during the term of this Agreement to devote substantially all of Executive’s business time, efforts, skills and abilities to the performance of Executive’s duties to the Company and to the furtherance of the Company's business.\n",
      "Output: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────────────────────────────────────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from autolabel import AutolabelDataset\n",
    "ds = AutolabelDataset(\"test.csv\", config=config)\n",
    "agent.plan(ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "dd703025-54d8-4349-b0d6-736d2380e966",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-06-14 13:31:32 openai INFO: error_code=None error_message='That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 9ff1d233ffc8b6842e6039eeb99aa39e in your message.)' error_param=None error_type=server_error message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 13:31:32 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 9ff1d233ffc8b6842e6039eeb99aa39e in your message.).\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-06-14 13:31:55 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88790 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 13:31:55 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88790 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 13:31:57 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 89075 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 13:31:57 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 89075 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 13:32:00 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88918 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 13:32:00 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88918 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 13:32:02 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88936 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 13:32:02 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88936 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 13:32:03 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88761 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 13:32:03 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 2.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88761 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 13:32:05 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88545 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 13:32:05 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88545 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 13:32:10 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88502 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 13:32:10 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88502 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 13:32:13 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88479 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 13:32:13 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88479 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 13:32:14 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88004 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 13:32:14 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88004 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 13:32:16 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 89399 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 13:32:16 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 89399 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-06-14 13:32:17 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 89085 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 13:32:17 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 2.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 89085 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 13:32:19 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88803 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 13:32:19 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88803 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 13:32:20 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88534 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 13:32:20 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 2.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88534 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 13:32:29 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88764 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 13:32:29 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88764 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 13:32:33 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87941 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 13:32:33 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87941 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 13:32:34 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87606 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 13:32:34 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 2.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87606 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 13:32:36 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88549 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 13:32:36 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88549 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Actual Cost: 0.2181\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> support </span>┃<span style=\"font-weight: bold\"> threshold </span>┃<span style=\"font-weight: bold\"> accuracy </span>┃<span style=\"font-weight: bold\"> completion_rate </span>┃\n",
       "┡━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 100     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> -inf      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.71     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0             </span>│\n",
       "└─────────┴───────────┴──────────┴─────────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1msupport\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mthreshold\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1maccuracy\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mcompletion_rate\u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m100    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m-inf     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.71    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "└─────────┴───────────┴──────────┴─────────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Total number of failures: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Total number of failures: \u001b[1;36m0\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# now, do the actual labeling\n",
    "ds = agent.run(ds, max_items=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "278e12eb",
   "metadata": {},
   "source": [
    "We are at 71% accuracy when labeling the first 100 examples. Let's see if we can use confidence scores to improve accuracy further by removing the less confident examples from our labeled set."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d7645ab",
   "metadata": {},
   "source": [
    "### Compute confidence scores\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "d2ae2d77",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Start computing confidence scores (using Refuel's LLMs)\n",
    "os.environ['REFUEL_API_KEY'] = 'xxxxxxxxxxxxxxxxx'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "5fbc1264",
   "metadata": {},
   "outputs": [],
   "source": [
    "# set `compute_confidence` -> True\n",
    "config[\"model\"][\"compute_confidence\"] = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "1998f5e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "agent = LabelingAgent(config=config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "119e6f22",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-06-14 15:03:28 langchain.embeddings.openai WARNING: Retrying langchain.embeddings.openai.embed_with_retry.<locals>._embed_with_retry in 4.0 seconds as it raised APIConnectionError: Error communicating with OpenAI: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response')).\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┌──────────────────────────┬─────────┐\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Total Estimated Cost     </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> $8.334  </span>│\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Number of Examples       </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> 2000    </span>│\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Average cost per example </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> $0.0042 </span>│\n",
       "└──────────────────────────┴─────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┌──────────────────────────┬─────────┐\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mTotal Estimated Cost    \u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m$8.334 \u001b[0m\u001b[1;32m \u001b[0m│\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mNumber of Examples      \u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m2000   \u001b[0m\u001b[1;32m \u001b[0m│\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mAverage cost per example\u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m$0.0042\u001b[0m\u001b[1;32m \u001b[0m│\n",
       "└──────────────────────────┴─────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────────── </span>Prompt Example<span style=\"color: #00ff00; text-decoration-color: #00ff00\"> ──────────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────────── \u001b[0mPrompt Example\u001b[92m ──────────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You are an expert at understanding legal contracts. Your job is to correctly classify legal provisions in contracts into one of the following categories.\n",
      "Categories:Agreements\n",
      "Amendments\n",
      "Anti-Corruption Laws\n",
      "Applicable Laws\n",
      "Approvals\n",
      "Arbitration\n",
      "Assignments\n",
      "Assigns\n",
      "Authority\n",
      "Authorizations\n",
      "Base Salary\n",
      "Benefits\n",
      "Binding Effects\n",
      "Books\n",
      "Brokers\n",
      "Capitalization\n",
      "Change In Control\n",
      "Closings\n",
      "Compliance With Laws\n",
      "Confidentiality\n",
      "Consent To Jurisdiction\n",
      "Consents\n",
      "Construction\n",
      "Cooperation\n",
      "Costs\n",
      "Counterparts\n",
      "Death\n",
      "Defined Terms\n",
      "Definitions\n",
      "Disability\n",
      "Disclosures\n",
      "Duties\n",
      "Effective Dates\n",
      "Effectiveness\n",
      "Employment\n",
      "Enforceability\n",
      "Enforcements\n",
      "Entire Agreements\n",
      "Erisa\n",
      "Existence\n",
      "Expenses\n",
      "Fees\n",
      "Financial Statements\n",
      "Forfeitures\n",
      "Further Assurances\n",
      "General\n",
      "Governing Laws\n",
      "Headings\n",
      "Indemnifications\n",
      "Indemnity\n",
      "Insurances\n",
      "Integration\n",
      "Intellectual Property\n",
      "Interests\n",
      "Interpretations\n",
      "Jurisdictions\n",
      "Liens\n",
      "Litigations\n",
      "Miscellaneous\n",
      "Modifications\n",
      "No Conflicts\n",
      "No Defaults\n",
      "No Waivers\n",
      "Non-Disparagement\n",
      "Notices\n",
      "Organizations\n",
      "Participations\n",
      "Payments\n",
      "Positions\n",
      "Powers\n",
      "Publicity\n",
      "Qualifications\n",
      "Records\n",
      "Releases\n",
      "Remedies\n",
      "Representations\n",
      "Sales\n",
      "Sanctions\n",
      "Severability\n",
      "Solvency\n",
      "Specific Performance\n",
      "Submission To Jurisdiction\n",
      "Subsidiaries\n",
      "Successors\n",
      "Survival\n",
      "Tax Withholdings\n",
      "Taxes\n",
      "Terminations\n",
      "Terms\n",
      "Titles\n",
      "Transactions With Affiliates\n",
      "Use Of Proceeds\n",
      "Vacations\n",
      "Venues\n",
      "Vesting\n",
      "Waiver Of Jury Trials\n",
      "Waivers\n",
      "Warranties\n",
      "Withholdings\n",
      "\n",
      "\n",
      "You will return the answer with just one element: \"the correct label\"\n",
      "\n",
      "Some examples with their output answers are provided below:\n",
      "\n",
      "Example: The Company hereby employs Executive and Executive accepts employment under the terms and conditions of this Agreement.\n",
      "Output: Employment\n",
      "\n",
      "Example: This Agreement, including its attached Exhibit A, constitutes the entire employment agreement between Executive and the Company regarding the terms and conditions of his employment. This Agreement supersedes all prior negotiations, representations or agreements between Executive and the Company, whether written or oral, concerning Executive’s employment.\n",
      "Output: Entire Agreements\n",
      "\n",
      "Example: Executive agrees that during the Employment Period, Executive will devote his/her full business time, energies and talents to serving as the Senior Vice President Community Banking of the Employer, at the direction of the Chief Executive Officer of the Employwer  (the “ CEO ”).  Executive shall have such duties and responsibilities as may be assigned to Executive from time to time by the CEO, which duties and responsibilities shall be commensurate with Executive’s position, shall perform all duties assigned to Executive faithfully and efficiently, subject to the direction of the CEO, and shall have such authorities and powers as are inherent to the undertakings applicable to Executive’s position and necessary to carry out the responsibilities and duties required of Executive hereunder.  Executive will perform the duties required by this Agreement at the Company’s principal place of business unless the nature of such duties requires otherwise.  Notwithstanding the foregoing, during the Employment Period, Executive may devote reasonable time to activities other than those required under this Agreement, including activities of a charitable, educational, religious or similar nature (including professional associations) to the extent such activities do not, in the reasonable judgment of the CEO, inhibit, prohibit, interfere with or conflict with Executive’s duties under this Agreement or conflict in any material way with the business of the Employer and its Affiliates; provided, however, that Executive shall not serve on the board of directors of any business (other than the Employer or its Affiliates) or hold any other position with any business without receiving the prior written consent of the CEO.\n",
      "Output: Duties\n",
      "\n",
      "Example: Executive represents that Executive's employment by the Company and the performance by Executive of her obligations under this Agreement do not, and shall not, breach any agreement, including, but not limited to, any agreement that obligates him to keep in confidence any trade secrets or confidential or proprietary information of her or of any other party, to perform services for any other party or to refrain from competing, directly or indirectly, with the business of any other party. Executive shall not disclose to the Company or use any trade secrets or confidential or proprietary information of any other party.\n",
      "Output: Representations\n",
      "\n",
      "Now I want you to label the following example:\n",
      "Example: Executive agrees to be employed with the Company, and the Company agrees to employ Executive, during the Term and on the terms and conditions set forth in this Agreement. Executive agrees during the term of this Agreement to devote substantially all of Executive’s business time, efforts, skills and abilities to the performance of Executive’s duties to the Company and to the furtherance of the Company's business.\n",
      "Output: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────────────────────────────────────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from autolabel import AutolabelDataset\n",
    "ds = AutolabelDataset(\"data/ledgar/test.csv\", config=config)\n",
    "agent.plan(ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "63c74705",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-06-14 15:04:00 autolabel.labeler INFO: Task run already exists.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">There is an existing task with following details: <span style=\"color: #808000; text-decoration-color: #808000\">id</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'3001625978'</span> <span style=\"color: #808000; text-decoration-color: #808000\">created_at</span>=<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">datetime</span><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">.datetime</span><span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2023</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">14</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">14</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">42</span>,\n",
       "<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">51</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">302325</span><span style=\"font-weight: bold\">)</span> <span style=\"color: #808000; text-decoration-color: #808000\">task_id</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'0890358f0f66b2f3e1284f6a180dc84a'</span> <span style=\"color: #808000; text-decoration-color: #808000\">dataset_id</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'a7f684e9cdafe3181a12b46035965fbe'</span> \n",
       "<span style=\"color: #808000; text-decoration-color: #808000\">current_index</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">100</span> <span style=\"color: #808000; text-decoration-color: #808000\">output_file</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'test_labeled.csv'</span> <span style=\"color: #808000; text-decoration-color: #808000\">status</span>=<span style=\"font-weight: bold\">&lt;</span><span style=\"color: #ff00ff; text-decoration-color: #ff00ff; font-weight: bold\">TaskStatus.ACTIVE:</span><span style=\"color: #000000; text-decoration-color: #000000\"> </span><span style=\"color: #008000; text-decoration-color: #008000\">'active'</span><span style=\"font-weight: bold\">&gt;</span> <span style=\"color: #808000; text-decoration-color: #808000\">error</span>=<span style=\"color: #800080; text-decoration-color: #800080; font-style: italic\">None</span> <span style=\"color: #808000; text-decoration-color: #808000\">metrics</span>=<span style=\"color: #800080; text-decoration-color: #800080; font-style: italic\">None</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "There is an existing task with following details: \u001b[33mid\u001b[0m=\u001b[32m'3001625978'\u001b[0m \u001b[33mcreated_at\u001b[0m=\u001b[1;35mdatetime\u001b[0m\u001b[1;35m.datetime\u001b[0m\u001b[1m(\u001b[0m\u001b[1;36m2023\u001b[0m, \u001b[1;36m6\u001b[0m, \u001b[1;36m14\u001b[0m, \u001b[1;36m14\u001b[0m, \u001b[1;36m42\u001b[0m,\n",
       "\u001b[1;36m51\u001b[0m, \u001b[1;36m302325\u001b[0m\u001b[1m)\u001b[0m \u001b[33mtask_id\u001b[0m=\u001b[32m'0890358f0f66b2f3e1284f6a180dc84a'\u001b[0m \u001b[33mdataset_id\u001b[0m=\u001b[32m'a7f684e9cdafe3181a12b46035965fbe'\u001b[0m \n",
       "\u001b[33mcurrent_index\u001b[0m=\u001b[1;36m100\u001b[0m \u001b[33moutput_file\u001b[0m=\u001b[32m'test_labeled.csv'\u001b[0m \u001b[33mstatus\u001b[0m=\u001b[1m<\u001b[0m\u001b[1;95mTaskStatus.ACTIVE:\u001b[0m\u001b[39m \u001b[0m\u001b[32m'active'\u001b[0m\u001b[1m>\u001b[0m \u001b[33merror\u001b[0m=\u001b[3;35mNone\u001b[0m \u001b[33mmetrics\u001b[0m=\u001b[3;35mNone\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluating the existing task<span style=\"color: #808000; text-decoration-color: #808000\">...</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluating the existing task\u001b[33m...\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metric: auroc: 0.5\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> support </span>┃<span style=\"font-weight: bold\"> threshold </span>┃<span style=\"font-weight: bold\"> accuracy </span>┃<span style=\"font-weight: bold\"> completion_rate </span>┃\n",
       "┡━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 100     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> -inf      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.71     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0             </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 100     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.5       </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.71     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0             </span>│\n",
       "└─────────┴───────────┴──────────┴─────────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1msupport\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mthreshold\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1maccuracy\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mcompletion_rate\u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m100    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m-inf     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.71    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m100    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.5      \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.71    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "└─────────┴───────────┴──────────┴─────────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">100</span> examples have been labeled so far.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m100\u001b[0m examples have been labeled so far.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────── </span>Last Annotated Example<span style=\"color: #00ff00; text-decoration-color: #00ff00\"> ──────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────── \u001b[0mLast Annotated Example\u001b[92m ──────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\">Prompt</span>: </pre>\n"
      ],
      "text/plain": [
       "\u001b[1;34mPrompt\u001b[0m: "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You are an expert at understanding legal contracts. Your job is to correctly classify legal provisions in contracts into one of the following categories.\n",
      "Categories:Agreements\n",
      "Amendments\n",
      "Anti-Corruption Laws\n",
      "Applicable Laws\n",
      "Approvals\n",
      "Arbitration\n",
      "Assignments\n",
      "Assigns\n",
      "Authority\n",
      "Authorizations\n",
      "Base Salary\n",
      "Benefits\n",
      "Binding Effects\n",
      "Books\n",
      "Brokers\n",
      "Capitalization\n",
      "Change In Control\n",
      "Closings\n",
      "Compliance With Laws\n",
      "Confidentiality\n",
      "Consent To Jurisdiction\n",
      "Consents\n",
      "Construction\n",
      "Cooperation\n",
      "Costs\n",
      "Counterparts\n",
      "Death\n",
      "Defined Terms\n",
      "Definitions\n",
      "Disability\n",
      "Disclosures\n",
      "Duties\n",
      "Effective Dates\n",
      "Effectiveness\n",
      "Employment\n",
      "Enforceability\n",
      "Enforcements\n",
      "Entire Agreements\n",
      "Erisa\n",
      "Existence\n",
      "Expenses\n",
      "Fees\n",
      "Financial Statements\n",
      "Forfeitures\n",
      "Further Assurances\n",
      "General\n",
      "Governing Laws\n",
      "Headings\n",
      "Indemnifications\n",
      "Indemnity\n",
      "Insurances\n",
      "Integration\n",
      "Intellectual Property\n",
      "Interests\n",
      "Interpretations\n",
      "Jurisdictions\n",
      "Liens\n",
      "Litigations\n",
      "Miscellaneous\n",
      "Modifications\n",
      "No Conflicts\n",
      "No Defaults\n",
      "No Waivers\n",
      "Non-Disparagement\n",
      "Notices\n",
      "Organizations\n",
      "Participations\n",
      "Payments\n",
      "Positions\n",
      "Powers\n",
      "Publicity\n",
      "Qualifications\n",
      "Records\n",
      "Releases\n",
      "Remedies\n",
      "Representations\n",
      "Sales\n",
      "Sanctions\n",
      "Severability\n",
      "Solvency\n",
      "Specific Performance\n",
      "Submission To Jurisdiction\n",
      "Subsidiaries\n",
      "Successors\n",
      "Survival\n",
      "Tax Withholdings\n",
      "Taxes\n",
      "Terminations\n",
      "Terms\n",
      "Titles\n",
      "Transactions With Affiliates\n",
      "Use Of Proceeds\n",
      "Vacations\n",
      "Venues\n",
      "Vesting\n",
      "Waiver Of Jury Trials\n",
      "Waivers\n",
      "Warranties\n",
      "Withholdings\n",
      "\n",
      "\n",
      "You will return the answer with just one element: \"the correct label\"\n",
      "\n",
      "Some examples with their output answers are provided below:\n",
      "\n",
      "Example: The waiver by either party of a breach of any provision of this Agreement, or failure to insist upon strict compliance with the terms of this Agreement, shall not be deemed a waiver of any subsequent breach or relinquishment of any right or power under this Agreement.\n",
      "Output: Waivers\n",
      "\n",
      "Example: This Agreement and various rights and obligations arising hereunder shall inure to the benefit of and be binding upon the Parties hereto and their successors and permitted assigns.  Neither this Agreement nor any of the rights, interests, or obligations hereunder shall be transferred, delegated, or assigned (by operation of Law or otherwise) by any Party hereto without the prior written consent of the other Parties.\n",
      "Output: Assignments\n",
      "\n",
      "Example: No term or condition of the Plan shall be deemed to have been waived, nor shall there be an estoppel against the enforcement of any provision of the Plan, except by written instrument of the party charged with such waiver or estoppel. No such written waiver shall be deemed a continuing waiver unless specifically stated therein, and each such waiver shall operate only as to the specific term or condition waived and shall not constitute a waiver of such term or condition for the future or as to any act other than that specifically waived. Any waiver by any party of a breach of any provision of the Plan by another party shall not operate or be construed as a waiver by such party of any subsequent breach thereof.\n",
      "Output: Waivers\n",
      "\n",
      "Example: Although the Guarantors party hereto have been informed of the matters set forth herein and have agreed to the same, each such Guarantor understands, acknowledges and agrees that none of the Secured Parties has any obligations to inform such Guarantor of such matters in the future or to seek such Guarantor’s acknowledgment or agreement to future amendments, restatements, supplements, changes, modifications, waivers or consents, and nothing herein shall create such a duty.\n",
      "Output: Agreements\n",
      "\n",
      "Now I want you to label the following example:\n",
      "Example: The waiver by a Party hereto of any right hereunder shall not be deemed a waiver of any other right hereunder, whether of a similar nature or otherwise.\n",
      "Output: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\">Annotation</span>: </pre>\n"
      ],
      "text/plain": [
       "\u001b[1;34mAnnotation\u001b[0m: "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waivers\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────────────────────────────────────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Do you want to resume the task? <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">[y/n]</span>: </pre>\n"
      ],
      "text/plain": [
       "Do you want to resume the task? \u001b[1;35m[y/n]\u001b[0m: "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Deleted the existing task and starting a new one<span style=\"color: #808000; text-decoration-color: #808000\">...</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Deleted the existing task and starting a new one\u001b[33m...\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-06-14 15:08:29 openai INFO: error_code=None error_message='The server had an error while processing your request. Sorry about that!' error_param=None error_type=server_error message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 15:08:29 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised APIError: The server had an error while processing your request. Sorry about that! {\n",
      "  \"error\": {\n",
      "    \"message\": \"The server had an error while processing your request. Sorry about that!\",\n",
      "    \"type\": \"server_error\",\n",
      "    \"param\": null,\n",
      "    \"code\": null\n",
      "  }\n",
      "}\n",
      " 500 {'error': {'message': 'The server had an error while processing your request. Sorry about that!', 'type': 'server_error', 'param': None, 'code': None}} {'Date': 'Wed, 14 Jun 2023 22:08:29 GMT', 'Content-Type': 'application/json', 'Content-Length': '176', 'Connection': 'keep-alive', 'access-control-allow-origin': '*', 'openai-model': 'gpt-3.5-turbo-0301', 'openai-organization': 'refuel', 'openai-processing-ms': '331', 'openai-version': '2020-10-01', 'strict-transport-security': 'max-age=15724800; includeSubDomains', 'x-ratelimit-limit-requests': '3500', 'x-ratelimit-limit-tokens': '90000', 'x-ratelimit-remaining-requests': '3499', 'x-ratelimit-remaining-tokens': '88015', 'x-ratelimit-reset-requests': '17ms', 'x-ratelimit-reset-tokens': '1.322s', 'x-request-id': 'e3bc90e7e76a1d78bf936aabe0639728', 'CF-Cache-Status': 'DYNAMIC', 'Server': 'cloudflare', 'CF-RAY': '7d75e463c8402368-SJC', 'alt-svc': 'h3=\":443\"; ma=86400'}.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metric: auroc: 0.7217\n",
      "Actual Cost: 0.01\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> support </span>┃<span style=\"font-weight: bold\"> threshold </span>┃<span style=\"font-weight: bold\"> accuracy </span>┃<span style=\"font-weight: bold\"> completion_rate </span>┃\n",
       "┡━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 100     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> -inf      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.71     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0             </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1       </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9996    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.01            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 14      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9969    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.14            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 15      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9965    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9333   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.15            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 16      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9964    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9375   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.16            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 18      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9953    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8333   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.18            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 31      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9868    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9032   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.31            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 32      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9866    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.875    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.32            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 35      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9759    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8857   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.35            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 36      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9759    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8611   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.36            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 39      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9725    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8718   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.39            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 40      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.972     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.85     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.4             </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 43      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9635    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8605   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.43            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 44      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9558    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8409   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.44            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 48      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9539    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8542   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.48            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 49      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9476    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8367   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.49            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 51      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9422    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8431   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.51            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 52      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9342    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8269   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.52            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 56      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9226    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8393   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.56            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 57      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9201    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8246   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.57            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 58      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9167    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8276   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.58            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 59      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9158    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8136   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.59            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 62      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8988    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8226   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.62            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 63      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8985    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8095   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.63            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 68      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8268    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8235   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.68            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 70      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8003    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7             </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 73      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.755     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8082   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.73            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 76      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7465    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7763   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.76            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 77      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7354    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7792   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.77            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 78      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.734     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7692   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.78            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 82      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7119    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7805   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.82            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 86      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6721    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7442   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.86            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 88      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.5911    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.75     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.88            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 89      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.5795    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7416   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.89            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 90      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.5614    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7444   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9             </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 92      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.53      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7283   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.92            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 94      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.5223    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.734    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.94            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 97      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.3197    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7113   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.97            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 98      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.1733    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7143   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.98            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 99      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.0402    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7071   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.99            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 100     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.02      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.71     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0             </span>│\n",
       "└─────────┴───────────┴──────────┴─────────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1msupport\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mthreshold\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1maccuracy\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mcompletion_rate\u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m100    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m-inf     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.71    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m1      \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9996   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.01           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m14     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9969   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.14           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m15     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9965   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9333  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.15           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m16     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9964   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9375  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.16           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m18     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9953   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8333  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.18           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m31     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9868   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9032  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.31           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m32     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9866   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.875   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.32           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m35     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9759   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8857  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.35           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m36     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9759   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8611  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.36           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m39     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9725   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8718  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.39           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m40     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.972    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.85    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.4            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m43     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9635   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8605  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.43           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m44     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9558   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8409  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.44           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m48     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9539   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8542  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.48           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m49     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9476   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8367  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.49           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m51     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9422   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8431  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.51           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m52     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9342   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8269  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.52           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m56     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9226   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8393  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.56           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m57     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9201   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8246  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.57           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m58     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9167   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8276  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.58           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m59     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9158   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8136  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.59           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m62     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8988   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8226  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.62           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m63     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8985   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8095  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.63           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m68     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8268   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8235  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.68           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m70     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8003   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m73     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.755    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8082  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.73           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m76     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7465   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7763  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.76           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m77     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7354   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7792  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.77           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m78     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.734    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7692  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.78           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m82     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7119   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7805  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.82           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m86     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6721   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7442  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.86           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m88     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.5911   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.75    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.88           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m89     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.5795   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7416  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.89           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m90     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.5614   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7444  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m92     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.53     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7283  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.92           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m94     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.5223   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.734   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.94           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m97     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.3197   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7113  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.97           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m98     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.1733   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7143  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.98           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m99     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.0402   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7071  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.99           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m100    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.02     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.71    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "└─────────┴───────────┴──────────┴─────────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Total number of failures: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Total number of failures: \u001b[1;36m0\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ds = agent.run(ds, max_items=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e89ff664",
   "metadata": {},
   "source": [
    "Looking at the table above, we can see that if we set the confidence threshold at `0.755`, we are able to label at 80.82% accuracy and getting a completion rate of 73%. This means, we would ignore all the data points where confidence score is less than `0.755` (which would end up being around 27% of all samples). This would, however, guarantee a very high quality labeled dataset for us. "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.8 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "vscode": {
   "interpreter": {
    "hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
